﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Xml;
using Microsoft.VisualBasic.FileIO;
using System.Xml.Linq;
using System.Text.RegularExpressions;
using System.IO;

/* Universität Trier
 * Fachbereich II - Linguistische Datenverarbeitung
 * Bachelor of Arts
 * Computerlinguistik (HF), English Language and Linguistics (NF)
 * 
 * 
 * A Heuristic Approach to Anaphora Resolution
 * 
 * Bachelor-Arbeit
 * 15. November 2013
 * 
 * Name:			Christopher Michels
 * Matrikelnummer:	1007830
 * 
 * Adresse:			Universitätsring 8d
 * 
 * Telefon:			(06 51) 99 24 11 55
 * E-Mail:			s2chmich@uni-trier.de
 * 
 * Erstprüfer: 		Dr. Sven Naumann
 * Zweitprüfer:		Prof. Dr. Reinhard Köhler
 * 
 * Sources:
 * Biber, Douglas et al. (1999). Longman Grammar of Spoken and Wri􀆩en English. Studies in Language and Linguistics. Harlow, UK: Longman.
 * Mitkov, Ruslan (1998). “Robust pronoun resolution with limited knowledge”. In: Proceedings of the 36th Annual Meeting of the Association for Computational Linguistics and 17th International Conference on Computational Linguistics. Vol. 2. ACL '98. Montreal, Canada: Association for Computational Linguistics, 869–875.
 * Mitkov, Ruslan (2000). “Pronoun Resolution: The Practical Alternative”. In: Corpus-based and Computational Approaches to Discourse Anaphora. Ed. by Simon Botley and Anthony Mark McEnery. Studies in Corpus Linguistics. J. Benjamins Pub. Co.
 * Mitkov, Ruslan (2002). Anaphora resolution. Studies in Language and Linguistics. London, UK: Longman.
 * Santorini, Beatrice (1991). Part-of-Speech Tagging Guidelines for the Penn Treebank Project. URL: http://www.cis.uni-muenchen.de/~schmid/tools/TreeTagger/data/Penn-Treebank-Tagset.pdf (visited on 11/06/2013).
 * Schmid, Helmut (1994). “Probabilistic Part-of-Speech Tagging Using Decision Trees”. In: Proceedings of the International Conference on New Methods in Language Processing. Manchester, UK.
 * Schmid, Helmut (1995). “Improvements In Part-of-Speech Tagging With an Application To German”. In: Proceedings of the ACL SIGDAT-Workshop. Dublin, Ireland.
 * Textalyser (2004). Bernhard Huber Internet Engineering Company. URL: http://textalyser.net (visited on 10/23/2013).
 */

namespace KPAR
{
    /* This class can create the XML file from the TreeTagger
     * chunker output, and it also acts as an interface for
     * accessing that file or for saving information.
     */
    static class AccessXML
    {
        /* Saves the location for the XML file.
         */
        public static string OutputFilePathXML { get; set; }

        /* Creates an XML file from the ouput file of the TreeTagger chunker.
         * Input is the file path of the chunker output file (type: string).
         */
        public static void CreateXML(string chunkOuputFilePath)
        {
            string dirName = Path.GetDirectoryName(chunkOuputFilePath);
            string fileNameWithoutEx = Path.GetFileNameWithoutExtension(chunkOuputFilePath);
            OutputFilePathXML = dirName + "\\KPAR_" + fileNameWithoutEx + ".xml";

            //Variables used to create valid structure and necessary attributes. 
            int headingID = 0;

            int sectionID = 0;
            int paragraphID = 0;
            int sentenceID = 0;

            int tokenID = 0;

            bool hasParagraph = false;
            bool belongsToHeading = false;
            bool newSentence = false;
            bool withinPC = false;

            //Reads the output file as a TSV file.
            using (TextFieldParser fieldParser = new TextFieldParser(chunkOuputFilePath, Encoding.GetEncoding(1252)))
            {
                fieldParser.TextFieldType = FieldType.Delimited;
                fieldParser.SetDelimiters("\t");

                //Creates an XML document with indentation. 
                using (XmlWriter chunksToXML = XmlWriter.Create(OutputFilePathXML, new XmlWriterSettings { Indent = true }))
                {
                    //Start the XML document with its root element.
                    chunksToXML.WriteStartDocument();
                    chunksToXML.WriteStartElement("text");

                    chunksToXML.WriteStartElement("section");
                    chunksToXML.WriteAttributeString("id", "" + ++sectionID);

                    //Iterate over the lines in the TSV file.
                    while (!fieldParser.EndOfData)
                    {
                        //Read the current line.
                        //Create a string array from the fields found in that line.
                        //Move the pointer to the next line.
                        string[] line = fieldParser.ReadFields();

                        if (line[0].StartsWith("<"))
                        {
                            #region ChunkerElementHandling

                            //End chunker elements and make sure prepositional environments are limited properly.
                            if (line[0].StartsWith("</"))
                            {
                                if (line[0].StartsWith("</PC"))
                                {
                                    withinPC = false;
                                }
                                chunksToXML.WriteEndElement();
                            }
                            else
                            {
                                //Start a new sentence element where necessary.
                                //IDs have subsections delimited by ":".
                                if (newSentence)
                                {
                                    chunksToXML.WriteStartElement("sentence");
                                    chunksToXML.WriteAttributeString("id", sectionID + ":" + paragraphID + ":" + ++sentenceID);
                                    newSentence = false;
                                }

                                tokenID++;

                                //Start the different chunker elements appropriately.
                                switch (line[0].Substring(1, line[0].Length - 2))
                                {
                                    case "CONJC":
                                        chunksToXML.WriteStartElement("CONJC");
                                        break;

                                    case "INTJ":
                                        chunksToXML.WriteStartElement("INTJ");
                                        break;

                                    case "LST":
                                        chunksToXML.WriteStartElement("LST");
                                        break;

                                    case "PRT":
                                        chunksToXML.WriteStartElement("PRT");
                                        break;

                                    case "VC":
                                        chunksToXML.WriteStartElement("VC");
                                        break;

                                    //Start a noun chunk element and add important attributes. 
                                    //The attributes "number" and "gender" are initialised with "indisc" (indiscernible).
                                    //The attribute "corefInfo" is initialised with "unknown".
                                    case "NC":
                                        chunksToXML.WriteStartElement("NC");
                                        chunksToXML.WriteAttributeString("number", "indisc");
                                        chunksToXML.WriteAttributeString("gender", "indisc");
                                        chunksToXML.WriteAttributeString("prepositional", withinPC.ToString());
                                        chunksToXML.WriteAttributeString("corefInfo", "unknown");
                                        break;

                                    case "PC":
                                        withinPC = true;
                                        chunksToXML.WriteStartElement("PC");
                                        break;

                                    case "ADJC":
                                        chunksToXML.WriteStartElement("ADJC");
                                        break;

                                    case "ADVC":
                                        chunksToXML.WriteStartElement("ADVC");
                                        break;
                                }

                                //A heading element does not have a partial ID for its paragraph but "H" instead. 
                                if (belongsToHeading)
                                {
                                    chunksToXML.WriteAttributeString("id", sectionID + ":H:" + headingID + ":" + tokenID);
                                }
                                else
                                {
                                    chunksToXML.WriteAttributeString("id", sectionID + ":" + paragraphID + ":" + sentenceID + ":" + tokenID);
                                }
                            }
                            #endregion
                        }

                        else
                        {
                            #region TaggerElementHandling

                            //Look at the text of the current token.
                            switch (line[0])
                            {
                                //Heading start is indicated manually by the symbol "~".
                                case "~":
                                    #region HeadingHandling
                                    //Does the latest heading already have a paragraph? 
                                    if (hasParagraph)
                                    {
                                        //Then start a new section and end the previous paragraph and ...
                                        chunksToXML.WriteEndElement();
                                        // ... the previous section.
                                        chunksToXML.WriteEndElement();

                                        hasParagraph = false;
                                        belongsToHeading = true;
                                        newSentence = false;

                                        chunksToXML.WriteStartElement("section");
                                        chunksToXML.WriteAttributeString("id", "" + ++sectionID);

                                        chunksToXML.WriteStartElement("heading");
                                        chunksToXML.WriteAttributeString("id", sectionID + ":" + ++headingID);
                                    }
                                    else
                                    {
                                        //Handle multiple subsequent headings.
                                        if (belongsToHeading)
                                        {
                                            chunksToXML.WriteEndElement();
                                        }
                                        //Add the current heading.
                                        belongsToHeading = true;
                                        chunksToXML.WriteStartElement("heading");
                                        chunksToXML.WriteAttributeString("id", sectionID + ":" + ++headingID);
                                    }
                                    #endregion
                                    break;

                                //Paragraph start is indicated manually by the symbol "§".
                                case "§":
                                    #region ParagraphHandling
                                    //Save changes in structure.
                                    hasParagraph = true;
                                    belongsToHeading = false;

                                    //End the previous paragraph or heading and start a paragraph.
                                    chunksToXML.WriteEndElement();
                                    chunksToXML.WriteStartElement("paragraph");
                                    chunksToXML.WriteAttributeString("id", sectionID + ":" + ++paragraphID);

                                    //Also start the first sentence of the paragraph.
                                    //Save that change in structure.
                                    chunksToXML.WriteStartElement("sentence");
                                    chunksToXML.WriteAttributeString("id", sectionID + ":" + paragraphID + ":" + ++sentenceID);
                                    newSentence = false;
                                    #endregion
                                    break;

                                //Default mode of the XMLWriter is to handle tokens.
                                default:
                                    #region TokenHandling
                                    //If required, start a new sentence first.
                                    //Save that change in structure.
                                    if (newSentence)
                                    {
                                        chunksToXML.WriteStartElement("sentence");
                                        chunksToXML.WriteAttributeString("id", sectionID + ":" + paragraphID + ":" + ++sentenceID);
                                        newSentence = false;
                                    }

                                    //Start the token element.
                                    chunksToXML.WriteStartElement("token");

                                    //A heading token does not have a partial ID for its paragraph but "H" instead.
                                    if (belongsToHeading)
                                    {
                                        chunksToXML.WriteAttributeString("id", sectionID + ":H:" + headingID + ":" + ++tokenID);
                                    }
                                    else
                                    {
                                        chunksToXML.WriteAttributeString("id", sectionID + ":" + paragraphID + ":" + sentenceID + ":" + ++tokenID);
                                    }

                                    //Extract the POS info and the lemma of the token from the remaining fields of the current line.
                                    if (line.Count() > 1)
                                    {
                                        chunksToXML.WriteAttributeString("pos", line[1]);
                                        chunksToXML.WriteAttributeString("lemma", line[2]);

                                        //Distinguish conjunctions and prepositions ("IN") as well as prepositional "to" and to-infinitve "to" ("TO").
                                        if (line[1].StartsWith("IN") || line[1].Equals("TO"))
                                            chunksToXML.WriteAttributeString("prepositional", withinPC.ToString());

                                        //Save the text value of the token and end the token element.
                                        chunksToXML.WriteString(line[0]);
                                        chunksToXML.WriteEndElement();

                                        //If a sentence ends, end the current sentence element.
                                        //Save that change in structure.
                                        if (!belongsToHeading && line[1].Equals("SENT"))
                                        {
                                            chunksToXML.WriteEndElement();
                                            newSentence = true;
                                        }
                                    }
                                    else
                                    {
                                        chunksToXML.WriteAttributeString("pos", line[0]);
                                        chunksToXML.WriteAttributeString("lemma", line[0]);
                                        //Save the text value of the token and end the token element.
                                        chunksToXML.WriteString(line[0]);
                                        chunksToXML.WriteEndElement();
                                    }

                                    #endregion
                                    break;
                            }
                            #endregion
                        }
                    }

                    //End the XML document with the end element of its root.
                    chunksToXML.WriteEndElement();
                    chunksToXML.WriteEndDocument();
                }
                AssignNounGenderAndNumber();
            }
        }

        /* Sets the value for gender and number of each NC if discernible. 
         */
        private static void AssignNounGenderAndNumber()
        {
            //Load the XML file for the current text file.
            XElement doc = XElement.Load(OutputFilePathXML);

            //Get all pronoun strings with an NC as their parent.
            IEnumerable<XElement> ncPronouns = 
                from p in doc.Descendants("token")
                where (Anaphora.PronounStrings.Contains(p.Value.ToLower())
                        && p.Parent.Name == "NC")
                select p;

            //Regular pronoun-NC IDs are stored here. 
            List<string> regularPronounIDs = new List<string>();

            //Set number and gender for all these pronouns.
            foreach (XElement p in ncPronouns)
            {
                string genderVal = "";
                string numberVal = "";

                //Only remember pronoun strings who do not have any elements
                //following them. If they do:
                //If the string is an attributively used possessive pronoun, prepare
                //the NC for 2 gender/number values.
                if (p.ElementsAfterSelf().FirstOrDefault() == null
                    || Anaphora.AttrPossStrings.Contains(p.Value.ToLower()))
                {
                    if (Anaphora.AttrPossStrings.Contains(p.Value.ToLower()))
                    {
                        genderVal = ":";
                        numberVal = ":";
                    }
                    else
                    {
                        regularPronounIDs.Add(p.Parent.SplitID().Last());
                    }

                    if (Anaphora.MascPronounStrings.Contains(p.Value.ToLower()))
                        genderVal += "masc";
                    else if (Anaphora.FemiPronounStrings.Contains(p.Value.ToLower()))
                        genderVal += "femi";
                    else if (Anaphora.NeutPronounStrings.Contains(p.Value.ToLower()))
                        genderVal += "neut";
                    else
                        genderVal += "indisc";

                    if (Anaphora.SgPronounStrings.Contains(p.Value.ToLower()))
                        numberVal += "sg";
                    else if (Anaphora.PlPronounStrings.Contains(p.Value.ToLower()))
                        numberVal += "pl";
                    else
                        numberVal += "indisc";

                    p.Parent.Attribute("gender").Value = genderVal;
                    p.Parent.Attribute("number").Value = numberVal;

                }
            }

            //Save changes to the XML file for the current text file.
            doc.Save(OutputFilePathXML);

            //Get all NCs.
            IEnumerable<XElement> NPs =
                from noun in doc.Descendants("NC")
                select noun;

            foreach (XElement np in NPs)
            {                
                if (!regularPronounIDs.Contains(np.SplitID().Last()))
                {
                    string genderVal = "indisc";
                    string numberVal = "indisc";

                    //Identify the last token.
                    XElement lastToken = np.Descendants("token").Last();

                    if (lastToken.Attribute("pos").Value.StartsWith("N"))
                    {
                        //Handle NCs tagged with "NP*", exclude heading NCs
                        if (lastToken.Attribute("pos").Value.StartsWith("NP"))
                        {
                            if (!lastToken.Attribute("id").Value.Contains("H"))
                            {
                                genderVal = "personal";
                            }
                            else
                            {
                                genderVal = "indisc";
                            }
                        }
                        else
                        {
                            genderVal = "neut";
                        }

                        if (Anaphora.NumberMismatchNouns.Contains(lastToken.Attribute("lemma").Value))
                            numberVal = "indisc";
                        else if (lastToken.Attribute("pos").Value.EndsWith("S"))
                            numberVal = "pl"; 
                        else
                            numberVal = "sg";
                    }
                    if (np.Descendants("token").Count() > 1)
                    {
                        //Try to find conjunctions as markers for plural.
                        foreach (XElement token in np.Descendants("token"))
                        {
                            if (token.Attribute("pos").Value.Equals("CC"))
                            {
                                genderVal = "indisc";
                                numberVal = "pl";
                            }
                        }
                    }

                    //Handle NC with 2 gender/number values due to an
                    //attributively used possessive pronoun.
                    if (np.Attribute("gender").Value.StartsWith(":"))
                    {
                        np.Attribute("gender").Value = genderVal + np.Attribute("gender").Value;
                        np.Attribute("number").Value = numberVal + np.Attribute("number").Value;
                    }
                    else
                    {
                        np.Attribute("gender").Value = genderVal;
                        np.Attribute("number").Value = numberVal;
                    }
                }
            }

            //Save changes to the XML file for the current text file.
            doc.Save(OutputFilePathXML);
        }

        /* Finds all pronoun strings listed in the class Anaphora of 
         * that type, filters them and returns a list of the remaining pronouns.
         */
        public static IEnumerable<XElement> AccessPronouns(List<string> pTypes)
        {
            //Load the XML file for the current text file.
            XElement doc = XElement.Load(OutputFilePathXML);

            //Find all pronouns with the strings specified in Anaphora.cs.
            IEnumerable<XElement> allPronouns =
                from pronoun in doc.Descendants("token")
                where Anaphora.PronounStrings.Contains(pronoun.Value.ToLower())
                select pronoun;

            //Redirected Output: Information about Pre-Processing 
            for (int i = 1; i <= 80; i++)
                Console.Write("-");
            Console.WriteLine();
            Console.WriteLine("Information - Pronouns:");
            for (int i = 1; i <= 80; i++)
                Console.Write("-");
            Console.WriteLine();
            Console.WriteLine("# Start:\t\t " + allPronouns.Count());

            List<XElement> filteredPronouns1 = new List<XElement>();

            //Not considered pronouns at all
            List<string> excludeIDs_PRON = new List<string>();
            //Actual determiners
            List<string> excludeIDs_DETS = new List<string>();
            //Usually deictic pronouns
            List<string> excludeIDs_DEIX_PRON = new List<string>();
            List<string> excludeIDs_DEIX_CAND = new List<string>();

            //Filter pronouns and add them to the appropriate list.
            foreach (XElement p in allPronouns)
            {
                if (Anaphora.PronounTypes.Contains(p.Attribute("pos").Value))
                {
                    if (Anaphora.DeicticPronounStrings.Contains(p.Value.ToLower()))
                    {
                        excludeIDs_DEIX_PRON.Add(p.Parent.SplitID().Last());
                        //Some noun phrases can still be candidates.
                        if (p.Parent.Descendants().Count() == 1)
                        {
                            excludeIDs_DEIX_CAND.Add(p.Parent.SplitID().Last());
                        }
                    }
                    else
                    {
                        if (p.Attribute("pos").Value.Equals("WDT")
                            || p.Attribute("pos").Value.Equals("DT"))
                        {
                            if (p.ElementsAfterSelf().FirstOrDefault() == null)
                            {
                                filteredPronouns1.Add(p);
                            }
                            else
                            {
                                excludeIDs_DETS.Add(p.Parent.SplitID().Last());
                            }
                        }
                        else
                        {
                            filteredPronouns1.Add(p);
                        }
                    }
                }
                else
                {
                    if (p.Parent.Name == "NC")
                        excludeIDs_PRON.Add(p.Parent.SplitID().Last());
                    else
                        excludeIDs_PRON.Add(p.SplitID().Last());
                }
            }

            Console.WriteLine("# Filter 1:\t\t" + (filteredPronouns1.Count() - allPronouns.Count()) + "\t\t"
                + "PRON:\t" + String.Join("~", excludeIDs_PRON)
                + "\n\t\t\t\t\t\tDEIX:\t" + String.Join("~", excludeIDs_DEIX_PRON)
                + "\n\t\t\t\t\t\tDETS:\t" + String.Join("~", excludeIDs_DETS));

            //Retrieve all relevant partial sentence IDs.
            List<string> pSentIDs = new List<string>();

            foreach (XElement p in allPronouns)
                pSentIDs.Add(p.SplitID()[2]);

            pSentIDs = pSentIDs.Distinct().ToList();
            //Console.WriteLine(pSentIDs.Count);

            IEnumerable<XElement> sentences =
                from sentence in doc.Descendants("sentence")
                let sentID = sentence.SplitID()[2]
                where pSentIDs.Contains(sentID)
                select sentence;

            //Create empty lists for the pronouns to be excluded. 
            List<string> excludeIDs = new List<string>();

            //pleonastic it
            List<string> excludeIDs_PLIT = new List<string>();
            //what-cleft what (for candidate filter only)
            List<string> excludeIDs_WHAT = new List<string>();

            //Create a string for each sentence, formatting its 
            //elements for the purpose of identifying constructions
            //which contain pleonastic or exophoric pronouns.
            #region ForeachFORMAT
            foreach (XElement s in sentences)
            {
                //Create a list for the formatted sentence elements.
                List<string> sentElemList = new List<string>();

                //Format each element.
                foreach (XNode n in s.Nodes())
                {
                    if (n is XElement)
                    {
                        XElement nAsXElem = (XElement)n;
                        XElement lastTokenOfNode =
                            nAsXElem.Descendants("token").LastOrDefault();

                        string nID = nAsXElem.SplitID().Last();

                        bool nodeIsToken = (lastTokenOfNode == null);

                        if (nodeIsToken)
                        {
                            sentElemList.Add(nAsXElem.Attribute("pos").Value);
                        }
                        else
                        {
                            string POSListString = nAsXElem.Name.ToString();

                            if (nAsXElem.Name == "NC")
                            {
                                if (nAsXElem.Descendants().Count() == 1)
                                {
                                    string singleTokenLemma =
                                        nAsXElem.Descendants().First().Attribute("lemma").Value;
                                    if (singleTokenLemma.ToLower() == "it")
                                    {
                                        POSListString = "it";
                                    }
                                    else if (singleTokenLemma.ToLower() == "what")
                                    {
                                        POSListString = "what";
                                    }
                                }
                                POSListString += ":" + nID;
                            }
                            else if (nAsXElem.Name == "VC")
                            {
                                //Find the last token of the verb, if possible.
                                IEnumerable<string> lemmasOfVC =
                                    from token in nAsXElem.Descendants("token")
                                    where token.Attribute("pos").Value.StartsWith("V") ||
                                            token.Attribute("pos").Value.Equals("MD")
                                    select token.Attribute("lemma").Value;
                                string lastlemmaOfVC =
                                    lemmasOfVC.LastOrDefault();

                                //Get the lemma of that token.
                                if (lastlemmaOfVC != null)
                                {
                                    if (lastlemmaOfVC == "be")
                                        POSListString = "be";
                                }
                            }
                            else if (nAsXElem.Name == "PC")
                            {
                                XElement embeddedNC =
                                    nAsXElem.Descendants("NC").FirstOrDefault();
                                if (embeddedNC != null)
                                {
                                    POSListString =
                                        "PC:NC:" + embeddedNC.SplitID().Last();
                                }
                            }
                            sentElemList.Add(POSListString);
                        }
                    }
                } 

                //Create a single string from the formatted elements.
                string stringToMatch = String.Join("_", sentElemList);

                //Identify what-clefts at the beginning of sentences.
                Match whatCleftmMatch = 
                    Regex.Match(stringToMatch, 
                        @"^What:([0-9]+)(_(PC:)?NC:[0-9]+)?(_be|_VC)" 
                        + @"(_PC|_ADJC)?(_be|_VC)", 
                        RegexOptions.IgnoreCase);

                //Identify it-clefts at the beginning of sentences.
                Match ItCleftmMatch =
                    Regex.Match(stringToMatch, @"^it:([0-9]+)(_be|_VC)"
                        + @"(_(PC:)?NC:[0-9]+|_IN/that|_ADJC)", 
                        RegexOptions.IgnoreCase);

                //Add the retrieved partial pronoun IDs to the respective lists.
                if (whatCleftmMatch.Success)
                {
                    excludeIDs_WHAT.Add(whatCleftmMatch.Groups[1].Value);
                }
                if (ItCleftmMatch.Success)
                {
                    excludeIDs_PLIT.Add(ItCleftmMatch.Groups[1].Value);
                }
            }
            #endregion

            //Filter the pronouns with the help of the concatenated lists.
            excludeIDs = excludeIDs_WHAT.Concat(excludeIDs_PLIT).ToList();

            IEnumerable<XElement> filteredPronouns2 =
                from p in filteredPronouns1
                where !excludeIDs.Contains(p.Parent.SplitID().Last())
                select p;

            //Redirected Output: Information about Pre-Processing 
            Console.WriteLine("# Filter 2:\t\t" 
                + (filteredPronouns2.Count() - filteredPronouns1.Count) + "\t\t"
                + "PLIT:\t" + String.Join("~", excludeIDs_PLIT) 
                + "\n\t\t\t\t\t\tWHAT:\t" + String.Join("~", excludeIDs_WHAT));
            Console.WriteLine("# End:\t\t\t " + filteredPronouns2.Count());
            for (int i = 1; i <= 80; i++)
                Console.Write("-");
            Console.WriteLine();

            //Pass the excludeIDs_WXYZ information on 
            //to the class Pronoun.
            Pronoun.IDsToExclude_PRON = excludeIDs_PRON;
            Pronoun.IDsToExclude_DEIX_PRON = excludeIDs_DEIX_PRON;
            Pronoun.IDsToExclude_DEIX_CAND = excludeIDs_DEIX_CAND;
            Pronoun.IDsToExclude_DETS = excludeIDs_DETS;
            Pronoun.IDsToExclude_PLIT = excludeIDs_PLIT;
            Pronoun.IDsToExclude_WHAT = excludeIDs_WHAT;

            //Return the list of validated anaphoric pronouns.
            return filteredPronouns2;
        }           

        /* Take the token XElement of a retrieved pronoun, assemble
         * relevant data and return an XElement storing the relevant
         * data.
         */
        public static XElement AccessPronounData(XElement pronoun)
        {
            //Load the XML file for the current text file.
            XElement doc = XElement.Load(OutputFilePathXML);

            //Create the storage XElement.
            XElement pronounData = new XElement("PronounData");

            //Get the array with the partial pronoun IDs.
            string[] pronID = pronoun.Parent.SplitID();

            //Save those partial IDs in variables.
            string pSectID = pronID[0];
            string pParaID = pronID[1];
            string pSentID = pronID[2];
            string pPronID = pronID[3];

            //Save some partial IDs as integer values for 
            //integer comparisons.
            int pSentID_Int = Int32.Parse(pSentID);
            int pPronID_Int = Int32.Parse(pPronID);


            //Find up to 3 relevant partial sentence IDs for
            //the current pronoun.
            List<string> pSentIDs = new List<string>();
            
            for (int i = pSentID_Int; (i > 0 && i >= pSentID_Int - 2); i--)
                pSentIDs.Add("" + i);

            //Retrieve the sentences with these partial IDs in the
            //same section as the pronoun.
            IEnumerable<XElement> sentences =
                from sentence in doc.Descendants("sentence")
                let sectID = sentence.SplitID()[0]
                let sentID = sentence.SplitID()[2]
                where sectID.Equals(pSectID) && pSentIDs.Contains(sentID)
                select sentence;

            //Find the relevant partial paragraph IDs for
            //the current pronoun. Uses the complete sentence ID
            //to retrieve them.
            List<string> pParaIDs = new List<string>();

            foreach (XElement s in sentences)
                pParaIDs.Add(s.SplitID()[1]);

            pParaIDs = pParaIDs.Distinct().ToList();
            //Console.WriteLine("Relevante Paragraphen: " + String.Join(", ", pParaIDs) + ";");

            //Find the headings of the section of the pronoun.
            IEnumerable<XElement> headings =
                from heading in doc.Descendants("heading")
                let sectID = heading.SplitID()[0]
                where sectID.Equals(pSectID)
                select heading;

            //Find the relevant paragraphs for the pronoun.
            IEnumerable<XElement> paragraphs =
                from paragraph in doc.Descendants("paragraph")
                let paraID = paragraph.SplitID()[1]
                where pParaIDs.Contains(paraID)
                select paragraph;

            //Find the NCs preceding the pronoun in the relevant sentences.
            IEnumerable<XElement> nouns =
                from noun in sentences.Descendants("NC")
                let nounID = Int32.Parse(noun.SplitID()[3])
                where nounID < pPronID_Int
                select noun;

            //Collect all relevant data with the the storage XElement.
            pronounData.Add(new XElement("pronoun", pronoun.Parent));
            pronounData.Add(new XElement("headings", headings));
            pronounData.Add(new XElement("paragraphs", paragraphs));
            pronounData.Add(new XElement("sentences", sentences));
            pronounData.Add(new XElement("nouns", nouns)); 

            return pronounData;
        }

        /* Takes a list of string pairs (type: string array, length is 2).
         * Index 0 is the pronoun's partial ID, index 1 is the candidate's 
         * partial ID. Saves the coreference information to the relevant
         * NC node's corefInfo attribute.
         */
        public static void SaveResults(List<string[]> anaphoraPairs)
        {
            //Load the XML file for the current text file.
            XElement doc = XElement.Load(OutputFilePathXML);

            //Count the number of anaphora to save.
            int anaphoraLinkCount = 0;

            foreach (string[] anaphoraPair in anaphoraPairs)
            {
                anaphoraLinkCount++;

                //Find the pronoun of the pair.
                XElement pronoun = 
                    (from p in doc.Descendants("NC")
                     where p.SplitID().Last().Equals(anaphoraPair[0])
                     select p).First();

                //If possible, find the antecedent of the pair ...
                if (anaphoraPair[1] != null)
                {
                    XElement antecendent =
                        (from a in doc.Descendants("NC")
                         where a.SplitID().Last().Equals(anaphoraPair[1])
                         select a).First();

                    //... and resolve the link. 
                    pronoun.Attribute("corefInfo").Value = 
                        "" + anaphoraLinkCount;

                    //Handle multiple links to the same antecedent candidate.
                    if (antecendent.Attribute("corefInfo").Value.Equals("unknown"))
                        antecendent.Attribute("corefInfo").Value = 
                            "" + anaphoraLinkCount;
                    else
                        antecendent.Attribute("corefInfo").Value = 
                            antecendent.Attribute("corefInfo").Value 
                            + ":" + anaphoraLinkCount;
                }
                //Otherwise, the corefInfo remains unresolved.
                else
                {
                    pronoun.Attribute("corefInfo").Value = ":UR";
                }
            }

            //Include information from other lists stored in the class pronoun.

            //Find the pronouns of the list TrivialIDs.
            IEnumerable<XElement> pronsTrivialIDs =
                from p in doc.Descendants("NC")
                where Pronoun.TrivialIDs.Contains(p.SplitID().Last())
                select p;

            foreach (XElement p in pronsTrivialIDs)
            {
                p.Attribute("corefInfo").Value = 
                    p.Attribute("corefInfo").Value 
                    + ":TRIV";
            }

            //Find the pronouns of the list NonCriticalIDs.
            IEnumerable<XElement> pronsNonCriticalIDs =
                from p in doc.Descendants("NC")
                where Pronoun.NonCriticalIDs.Contains(p.SplitID().Last())
                select p;

            foreach (XElement p in pronsNonCriticalIDs)
            {
                p.Attribute("corefInfo").Value = 
                    p.Attribute("corefInfo").Value 
                    + ":NCRI";
            }
            
            //Find the pronouns of the list IDsToExclude_DEIX.
            IEnumerable<XElement> pronsIDsToExclude_DEIX =
                from p in doc.Descendants("NC")
                where Pronoun.IDsToExclude_DEIX_PRON.Contains(p.SplitID().Last())
                select p;

            foreach (XElement p in pronsIDsToExclude_DEIX)
            {
                p.Attribute("corefInfo").Value = 
                    p.Attribute("corefInfo").Value
                    + ":EXCL:DEIX";
            }

            //Find the pronouns of the list IDsToExclude_DETS.
            IEnumerable<XElement> pronsIDsToExclude_DETS =
                from p in doc.Descendants("NC")
                where Pronoun.IDsToExclude_DETS.Contains(p.SplitID().Last())
                select p;

            foreach (XElement p in pronsIDsToExclude_DETS)
            {
                p.Attribute("corefInfo").Value = 
                    p.Attribute("corefInfo").Value
                    + ":EXCL:DETS";
            }

            //Find the pronouns of the list IDsToExclude_PLIT.
            IEnumerable<XElement> pronsIDsToExclude_PLIT =
                from p in doc.Descendants("NC")
                where Pronoun.IDsToExclude_PLIT.Contains(p.SplitID().Last())
                select p;

            foreach (XElement p in pronsIDsToExclude_PLIT)
            {
                p.Attribute("corefInfo").Value = 
                    p.Attribute("corefInfo").Value
                    + ":EXCL:PLIT";
            }

            //Find the pronouns of the list IDsToExclude_WHAT.
            IEnumerable<XElement> pronsIDsToExclude_WHAT =
                from p in doc.Descendants("NC")
                where Pronoun.IDsToExclude_WHAT.Contains(p.SplitID().Last())
                select p;

            foreach (XElement p in pronsIDsToExclude_WHAT)
            {
                p.Attribute("corefInfo").Value = 
                    p.Attribute("corefInfo").Value
                    + ":EXCL:WHAT";
            }

            //Save changes to the XML file for the current text file.

            if (File.Exists(OutputFilePathXML))
            {
                File.Delete(OutputFilePathXML);
            }

            doc.Save(OutputFilePathXML);
        }

        /* Creates a TXT file from the XML file.
         * Input is the destination file path for the TXT file (type: string).
         */
        public static void CreateTXT(string filePathXML, string txtOuputFilePath)
        {
            //Load the XML file for the current text file.
            XElement doc = XElement.Load(filePathXML);

            using (StreamWriter streamWriter = new StreamWriter(txtOuputFilePath))
            {
                streamWriter.WriteLine(HandleNodes(doc));
            }
        }

        /* Recursive method, formatting text of XElement parent for CreateTXT
         */
        private static string HandleNodes(XElement parent)
        {
            string output = "";

            foreach (XNode n in parent.Nodes())
            {
                if (n is XElement)
                {
                    XElement child = (XElement)n;
                    //output = output + "name:" + child.Name + ":name\n";

                    if (child.Name == "section")
                    {
                        output = output + "\n" + HandleNodes(child) + "---\n";
                    }
                    else if (child.Name == "heading")
                    {
                        output = output + "~" + HandleNodes(child) + "~\n";
                    }
                    else if (child.Name == "paragraph")
                    {
                        output = output + "\n§ " + HandleNodes(child) + "\n";
                    }
                    else if (child.Name == "sentence")
                    {
                        output = output + HandleNodes(child) + " ";
                    }
                    else if (child.Name == "NC")
                    {
                        if (!(child.Attribute("corefInfo").Value == "unknown"))
                            //|| child.Attribute("excluded").Value == ","))
                            output = output + " (" + HandleNodes(child) + "):" + child.Attribute("corefInfo").Value;
                        else
                            output = output + " " + HandleNodes(child);
                        output = output.TrimStart();
                    }

                    else if (child.Name == "token")
                    {
                        if (!(child.Attribute("pos").Value == "SENT"
                            || child.Attribute("pos").Value == ","))
                            output = output + " ";
                        output = output + child.Value;
                        output = output.TrimStart();
                    }
                    else
                    {
                        output = output + " " + HandleNodes(child);
                        output = output.TrimStart();
                    }
                }
                else
                {
                    output = output + "???:" + n + ":???\n";
                }
            }

            return output;
        }
    }
}
